In [4]:
%matplotlib inline
%config InlineBackend.figure_format='retina'
from IPython.display import Image, HTML
import matplotlib.pyplot as plt
import seaborn as sns
from itertools import cycle
import math
import datetime as dt
import pandas as pd
import numpy as np
import pytz
import matplotlib.dates as mpd
from mpl_toolkits.mplot3d import Axes3D
import scipy
from scipy import signal
from scipy.signal import medfilt
import numexpr as ne
from patsy import dmatrices
from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.cross_validation import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler, PolynomialFeatures, Normalizer, MinMaxScaler, Binarizer
from sklearn import metrics
from sklearn.cluster import AgglomerativeClustering, KMeans, DBSCAN, Birch, AffinityPropagation
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA, KernelPCA, MiniBatchSparsePCA, IncrementalPCA
from hdbscan import HDBSCAN
import statsmodels.api as sm
import random
import os
from enerpi.base import timeit
from enerpi.api import enerpi_data_catalog, enerpi_log
from enerpi.enerplot import write_fig_to_svg, tableau20
from enerpi.sunposition import sun_position
from prettyprinting import *
loginfo = enerpi_log(log_file=os.path.expanduser('~/ENERPIDATA/enerpi_study_temps.log'), extract_temps=True)
In [7]:
print_cyan(loginfo.count())
print_yellow(loginfo.tipo.value_counts())
print_magenta(loginfo['exec'].value_counts())
In [8]:
loginfo[loginfo.tipo == 'WARNING']
Out[8]:
In [106]:
temps = loginfo[loginfo.CPU.notnull()].drop(['tipo', 'no_red', 'debug_send'], axis=1)
temps[temps.temp.isnull()].head(10)
Out[106]:
In [14]:
temps.msg.str.slice(0, 9).value_counts()
Out[14]:
In [266]:
temp_comb = pd.DataFrame(temps[['CPU', 'GPU']].max(axis=1), columns=['temp']).resample('3s').mean().fillna(method='bfill')
temp_comb['fecha'] = temp_comb.index.date
print_info(temp_comb.head())
print_cyan(temp_comb.describe().T)
delta_temp = temp_comb.temp.max() - temp_comb.temp.min()
sample_temps = temp_comb.temp.iloc[1000:2000]
plt.figure(figsize=(18, 5))
plt.subplot(1, 3, 1)
plt.hist(temp_comb.temp, bins=int(round(delta_temp)), log=True)
plt.subplot(1, 3, 2)
plt.plot(sample_temps);
plt.subplot(1, 3, 3)
plt.plot(sample_temps.iloc[:200]);
In [267]:
plt.figure(figsize=(18, 10))
ax = sns.boxplot(x='fecha', y='temp', data=temp_comb, width=.9, fliersize=2, linewidth=1)
xticks = ax.get_xticks()
xticklabels = ax.get_xticklabels()
ax.set_xticks(xticks[::2])
ax.set_xticklabels(xticklabels[::2])
ax.grid()
ax.set_xlabel('')
ax.set_ylabel('Temperatura RPI (max(cpu, gpu)) (ºC)');
In [277]:
temp_hom = pd.DataFrame(temps[['CPU', 'GPU']].max(axis=1), columns=['temp']).resample('1s').mean().interpolate()
sample_temps = temp_hom.temp.iloc[:10000]
y = (sample_temps.rolling(15, center=True).mean() - sample_temps.mean()).dropna() #.resample('500ms').interpolate()
N = len(y)
frate = 1 #/sec
Hn = np.fft.fft(y)
freqs = np.fft.fftfreq(len(Hn), frate) * N * frate
amplitudes = np.abs(Hn)
df_fft = pd.DataFrame([amplitudes[:N//2], freqs[:N//2]]).T.rename(columns={0:'ampl', 1:'freq'}).sort_values(by='ampl', ascending=False)
idx = np.argmax(amplitudes)
freq_in_hertz = freqs[idx]
print(freq_in_hertz, idx)
plt.figure(figsize=(16, 10))
plt.semilogy(amplitudes[:N//2], alpha=.8)
df_fft.head(10)
Out[277]:
In [229]:
(pd.Series(np.fft.irfft(Hn)[:N], index=y.index)).iloc[:1000].plot(figsize=(18, 5), lw=1.5, color='r')
plt.show()
In [173]:
prueba = sample_temps.iloc[:2000]
ax = prueba.plot(figsize=(10, 5), lw=1, alpha=.2)
#pd.Series(medfilt(prueba, kernel_size=11), index=prueba.index, name='medfilt_7').plot(ax=ax, alpha=.5, lw=3)
prueba.rolling(9, center=True).mean().rename('medfilt_7').plot(ax=ax, alpha=.5, lw=3)
#pd.Series(medfilt(prueba, kernel_size=15), index=prueba.index, name='medfilt_15').plot(ax=ax, alpha=.5)
plt.legend();
In [27]:
timmings = loginfo[loginfo.msg.str.startswith('TIMEIT ')].dropna(axis=1, how='all')
timmings = timmings.join(timmings.msg.str.extract('TIMEIT (?P<func>.+) TOOK: (?P<took>\d{1,3}\.\d{1,3}) s', expand=True)
).drop(['tipo', 'msg'], axis=1)
timmings.took = timmings.took.astype(float)
gb_timmings = timmings.groupby(['exec', 'func'])
print_cyan(gb_timmings.describe(percentiles=[.1, .9]))
timmings.head(10)
Out[27]:
In [259]:
t_funcs = timmings.reset_index().set_index(['func', 'ts']).sort_index()
plt.figure(figsize=(18, 10))
for f in ['archive_periodic', 'update_catalog', '_distribute_data', '_make_index', ]:
s = t_funcs.loc[f, 'took'].rename(f[1:] if f.startswith('_') else f)
s.plot()
plt.ylim(0, 4)
plt.ylabel('Tiempo de ejecución (segundos)')
plt.xlabel('')
plt.title('Timmings de tareas')
plt.legend();
In [ ]:
In [ ]: